/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */

package amd.core.extraction;

import amd.core.data.ListTermFrequency;
import amd.core.*;

/**
 *
 * @author Latifah Dwi Nurachma
 */
public class TermFilter {
    /*
        Stoplist taken from http://nlp.cs.nyu.edu/GMA_files/resources/english.stoplist
    */
    private static String[] stoplist= {
        " ", "a", "able", "aboard", "about", "above", "according",
        "accordingly",	"across",	"actually",	"after",	"afterwards",	"again",
        "against",	"agin",	"ago",	"agreed-upon",	"ah",	"ain't",
        "alas",	"albeit",	"all",	"all-over",	"allow",	"allows",
        "almost",	"alone",	"along",	"along",	"alongside",	"already",
        "also",	"altho",	"although",	"always",	"am",	"amid",
        "amidst",	"among",	"amongst",	"an",	"and",	"another",
        "any",	"anybody",	"anyhow",	"anyone",	"anything",	"anyway",
        "anyways",	"anywhere",	"apart",	"appear",	"appreciate",	"appropriate",
        "are",	"aren't",	"around",	"as",	"a's",	"aside",
        "ask",	"asking",	"associated",	"astride",	"at",	"atop",
        "available",	"avec",	"away",	"awfully",	"back",	"be",
        "became",	"because",	"become",	"becomes",	"becoming",	"been",
        "before",	"beforehand",	"behind",	"being",	"believe",	"below",
        "beneath",	"beside",	"besides",	"best",	"better",	"between",
        "beyond",	"bi",	"both",	"brief",	"but",	"by",
        "ca",	"came",	"can",	"cannot",	"cant",	"can't",
        "cause",	"causes",	"causing",	"certain",	"certainly",	"change",
        "changes",	"clearly",	"c'mon",	"co",	"com",	"come",
        "comes",	"concerning",	"consequently",	"consider",	"considering",	"contain",
        "containing",	"contains",	"corresponding",	"could",	"couldn't",	"course",
        "c's",	"currently",	"de",	"definitely",	"des",	"described",
        "despite",	"did",	"didn't",	"different",	"do",	"does",
        "doesn't",	"doing",	"done",	"don't",	"down",	"downwards",
        "due",	"durin",	"during",	"each",	"edu",	"eg",
        "eh",	"either",	"else",	"elsewhere",	"en",	"enough",
        "entirely",	"especially",	"et",	"etc",	"even",	"ever",
        "every",	"everybody",	"everyone",	"everything",	"everywhere",	"ex",
        "exactly",	"example",	"except",	"far",	"fer",	"few",
        "fifth",	"first",	"followed",	"following",	"follows",	"for",
        "former",	"formerly",	"from",	"further",	"furthermore",	"get",
        "gets",	"getting",	"given",	"gives",	"go",	"goddamn",
        "goes",	"going",	"gone",	"goody",	"gosh",	"got",
        "gotten",	"greetings",	"had",	"hadn't",	"half",	"happens",
        "hardly",	"has",	"hasn't",	"have",	"haven't",	"having",
        "he",	"hell",	"hello",	"help",	"hence",	"her",
        "here",	"hereafter",	"hereby",	"herein",	"here's",	"hereupon",
        "hers",	"herself",	"he's",	"hey",	"hi",	"him",
        "himself",	"his",	"hither",	"ho",	"hopefully",	"how",
        "how",	"how",	"howbeit",	"however", "i","i'd",	"ie",
        "if",	"ignored",	"i'll",	"i'm",	"immediate",	"in",
        "inasmuch",	"inc",	"indeed",	"indicate",	"indicated",	"indicates",
        "inner",	"inside",	"insofar",	"instead",	"into",	"inward",
        "is",	"isn't",	"it",	"it'd",	"it'll",	"its",
        "it's",	"itself",	"i've",	"just",	"keep",	"keeps",
        "kept",	"know",	"known",	"knows",	"la",	"last",
        "lately",	"later",	"latter",	"latterly",	"le",	"least",
        "les",	"less",	"lest",	"let",	"let's",	"lieu",
        "like",	"like",	"liked",	"likely",	"little",	"look",
        "looking",	"looks",	"ltd",	"mainly",	"many",	"may",
        "maybe",	"me",	"mean",	"meanwhile",	"merely",	"might",
        "minus",	"more",	"moreover",	"most",	"mostly",	"Mr",
        "Mrs",	"Ms",	"much",	"must",	"my",	"myself",
        "name",	"namely",	"nd",	"near",	"near-by",	"nearer",
        "nearest",	"nearly",	"necessary",	"need",	"needs",	"neither",
        "never",	"nevertheless",	"new",	"next",	"no",	"nobody",
        "non",	"none",	"noone",	"nor",	"normally",	"not",
        "nothing",	"notwithstanding",	"novel",	"now",	"nowhere",	"obviously",
        "o'er",	"of",	"off",	"often",	"oh",	"ok",
        "okay",	"old",	"on",	"once",	"one",	"ones",
        "oneself",	"only",	"onto",	"or",	"other",	"others",
        "otherwise",	"ought",	"our",	"ours",	"ourselves",	"out",
        "outside",	"outta",	"over",	"over",	"overall",	"own",
        "particular",	"particularly",	"per",	"perhaps",	"placed",	"please",
        "plus",	"possible",	"presumably",	"probably",	"provides",	"que",
        "quite",	"qv",	"rather",	"rd",	"re",	"really",
        "reasonably",	"regarding",	"regardless",	"regards",	"relatively",	"respectively",
        "right",	"round",	"said",	"same",	"saw",	"say",
        "saying",	"says",	"se",	"second",	"secondly",	"see",
        "seeing",	"seem",	"seemed",	"seeming",	"seems",	"seen",
        "self",	"selves",	"sensible",	"sent",	"serious",	"seriously",
        "several",	"shall",	"she",	"should",	"shouldn't",	"since",
        "six",	"so",	"some",	"somebody",	"somehow",	"someone",
        "something",	"sometime",	"sometimes",	"somewhat",	"somewhere",	"soon",
        "sorry",	"specified",	"specify",	"specifying",	"still",	"sub",
        "such",	"sup",	"sure",	"take",	"taken",	"tell",
        "tends",	"th",	"than",	"thank",	"thanks",	"thanx",
        "that",	"thats",	"that's",	"the",	"their",	"theirs",
        "them",	"themselves",	"then",	"thence",	"there",	"thereafter",
        "thereby",	"therefore",	"therein",	"theres",	"there's",	"thereupon",
        "these",	"they",	"they'd",	"they'll",	"they're",	"they've",
        "thine",	"think",	"third",	"this",	"thorough",	"thoroughly",
        "those",	"thou",	"though",	"through",	"throughout",	"thru",
        "thus",	"till",	"to",	"together",	"too",	"took",
        "toward",	"toward",	"towardes",	"towards",	"towards",	"tried",
        "tries",	"truly",	"try",	"trying",	"t's",	"twice",
        "uh",	"un",	"under",	"underneath",	"unfortunately",	"unless",
        "unlike",	"unlikely",	"until",	"unto",	"unto",	"up",
        "upon",	"uppon",	"us",	"use",	"used",	"useful",
        "uses",	"using",	"usually",	"uucp",	"value",	"various",
        "ve",	"very",	"via",	"vis-a-vis",	"vis-à-vis",	"viz",
        "vs",	"w",	"want",	"wants",	"was",	"wasn't",
        "way",	"we",	"we'd",	"welcome",	"well",	"we'll",
        "went",	"were",	"we're",	"weren't",	"we've",	"what",
        "whatever",	"what's",	"whatsoever",	"when",	"whence",	"whenever",
        "where",	"whereafter",	"whereas",	"whereas",	"whereby",	"wherefore",
        "wherein",	"where's",	"whereupon",	"wherever",	"whether",	"which",
        "whichever",	"while",	"whither",	"who",	"whoever",	"whole",
        "whom",	"who's",	"whose",	"why",	"will",	"willing",
        "wish",	"with",	"withal",	"within",	"without",	"wonder",
        "won't",	"would",	"wouldn't",	"ye",	"yea",	"yeah",
        "yes",	"yet",	"yonder",	"you",	"you'd",	"you'll",
        "your",	"you're",	"yours",	"yourself",	"yourselves", "you've" 
    };

    public static ListTermFrequency filtering(ListTermFrequency terms){
		int i = 0, indexStoplist = 0;
        ListTermFrequency newTerm = new ListTermFrequency();
        
        while(i<terms.size()){
            if (!isStoplist(terms.get(i).getContent())){
                newTerm.add(terms.get(i));
            }
            i++;
        }
        return newTerm;
	}

    public static boolean isStoplist(String s){
        int i = 0;
        boolean isStoplist = false;
        int nStoplist = stoplist.length;

        while (i< nStoplist){
            if (s.equalsIgnoreCase(stoplist[i])){
                isStoplist = true;
            }
            i++;
        }
        return isStoplist;
    }
//    public static int getStoplistLength(){
//        return stoplist.length;
//    }
}
